package com.wanghang89.gmail.http.crawler;

import java.util.List;
import java.util.concurrent.TimeUnit;

import org.apache.log4j.Logger;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.wanghang89.gmail.bean.AdministraRegion;
import com.wanghang89.gmail.bean.Url;
import com.wanghang89.gmail.http.Downloader;
import com.wanghang89.gmail.http.EncodingConstants;
import com.wanghang89.gmail.util.MathUtils;
import com.wanghang89.gmail.util.db.AdministraRegionDbUtils;
import com.wanghang89.gmail.util.db.DBException;
import com.wanghang89.gmail.util.db.UrlDbUtils;

public class WeatherUrlCrawler implements EncodingConstants{

	private AdministraRegionDbUtils administraRegionDbUtils ;
	private UrlDbUtils urlDbUtils ;
	
	private final static String URL_PREFIX = "http://www.nmc.gov.cn";
	private final static Logger LOGGER = Logger.getLogger(WeatherUrlCrawler.class);
	
	public WeatherUrlCrawler()
	{
		
	}
	
	public void crawl()
	{
		try {
			crawlProvinceUrl();
			try {
				TimeUnit.SECONDS.sleep(MathUtils.random(60));
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
			crawlSubordinateUrl();
		} catch (DBException e) {
			e.printStackTrace();
		}
	}
		
	private void crawlProvinceUrl() throws DBException
	{		
		String ru = "http://www.nmc.gov.cn/publish/weather/capital.html";	
		Document doc = Downloader.downloadByGet(ru,GBK);
		String pars = "div[class=namecity] a";
		Elements pares = doc.select(pars);
		if (pares!=null&&!pares.isEmpty()) {
			int size = pares.size();
			int  type=1;
			for (int i = 0; i < size ; i++) {
				Element pare = pares.get(i);
				String name = pare.text();
				String href = URL_PREFIX+pare.attr("href");
				Document fdoc = Downloader.downloadByGet(href, GBK);
				Element e = fdoc.getElementById("iframe");
				href = URL_PREFIX+e.attr("src");
				System.out.println(name+":"+href);
				Url url = new Url();
				url.setHref(href);
				url.setRemark(name);
				url.setUrlTypeId(1);
				url.setStatus(1);
				Integer id = urlDbUtils.add(url);
				if (id>0) {
					AdministraRegion ar = administraRegionDbUtils.getByFuzzyNameAndGrade(name, type);
					if (ar!=null) {
						administraRegionDbUtils.updateUrlId(ar.getId(),id);
					}
				}
			}
		}
	}
	
	private void crawlSubordinateUrl() throws DBException
	{
		List<AdministraRegion> ars = administraRegionDbUtils.getByGrade(1);
		int size = ars.size();
		LOGGER.debug("size:"+size);
		for (int i = 0; i < size ; i++) {
			AdministraRegion ar = ars.get(i);
			List<AdministraRegion> subordinates = administraRegionDbUtils.getAllSubordinateById(ar.getId());
			Integer urlId = ar.getUrlId();
			LOGGER.debug("url id:"+urlId);
			if(urlId!=null) {
				Url url = urlDbUtils.getById(urlId);
				Document doc = Downloader.downloadByGet(url.getHref(), GBK);
				Elements es = doc.select("div[class=namecity] a");
				if (!es.isEmpty()) {
					int essize = es.size();
					for (int j = 0; j < essize; j++) {
						Element e = es.get(j);
						String href = URL_PREFIX+e.attr("href");
						String tn = e.html();
						Integer arId = getSubordinateId(subordinates,tn);
						if (arId!=-1) {
							Document fdoc2 = Downloader.downloadByGet(href, GBK);
							Elements ies = fdoc2.select("iframe[class=irw]");
							if (!ies.isEmpty()) {
								href = URL_PREFIX+ies.first().attr("src");
								Url url2 = new Url();
								url2.setHref(href);
								url2.setRemark(tn);
								url2.setStatus(1);
								url2.setUrlTypeId(1);
								Integer uId = urlDbUtils.add(url2);
								administraRegionDbUtils.updateUrlId(arId, uId);
							}
						}
						try {
							TimeUnit.SECONDS.sleep(MathUtils.random(10));
						} catch (InterruptedException e1) {
							e1.printStackTrace();
						}
					}
				}	
			}
		}
	}
	
	private static Integer getSubordinateId(List<AdministraRegion> subordinates , String name)
	{
		int size = subordinates.size();
		for (int i = 0; i < size ; i++) {
			AdministraRegion ar = subordinates.get(i);
			String arn = ar.getName();
			if (arn.startsWith(name)) {
				LOGGER.debug("name:"+name+";arname:"+arn);
				return ar.getId();
			}
		}
		return -1;
	}
	
	public AdministraRegionDbUtils getAdministraRegionDbUtils() {
		return administraRegionDbUtils;
	}

	public void setAdministraRegionDbUtils(
			AdministraRegionDbUtils administraRegionDbUtils) {
		this.administraRegionDbUtils = administraRegionDbUtils;
	}

	public UrlDbUtils getUrlDbUtils() {
		return urlDbUtils;
	}

	public void setUrlDbUtils(UrlDbUtils urlDbUtils) {
		this.urlDbUtils = urlDbUtils;
	}

	public static void main(String[] args) throws DBException {
		
		
		
//		String name = "北京";
//		String href = "";
//		AdministraRegion ar = administraRegionDbUtils.getByFuzzyNameAndGrade(name,1);
//		System.out.println(ar.getId()+"_____________");
//		LOGGER.debug("id:"+(ar!=null?(ar.getId()+"|"+ar.getName()):"null")+";name:"+name+";href:"+href);
		
//		String url = "http://www.nmc.gov.cn/publish/forecast/china.html";
//		Document doc = Downloader.downloadByGet(url, GBK);
//		System.out.println(doc.html());
		
		
//		String url1 = "http://www.nmc.gov.cn/publish/forecast/provices/ABJ.html";
//		Document doc = Downloader.downloadByGet(url1, GBK);
//		System.out.println(doc.html());
		
		
//		String url2 = "http://www.nmc.gov.cn/publish/weather/provice/ABJ_24.html";
//		Document doc2 = Downloader.downloadByGet(url2, GBK);
//		System.out.println(doc2.html());
		
//		String url3 = "http://www.nmc.gov.cn/publish/forecast/ABJ/beijing.html";
//		Document doc3 = Downloader.downloadByGet(url3, GBK);
//		System.out.println(doc3.html());
		
//		String url4 = "http://www.nmc.gov.cn/publish/forecast/ABJ/beijing_iframe.html";
//		Document doc4 = Downloader.downloadByGet(url4, GBK);
//		System.out.println(doc4.html());
		
//		String url5 = "http://www.nmc.gov.cn/publish/weather/capital.html";
//		Document doc5 = Downloader.downloadByGet(url5, GBK);
//		System.out.println(doc5.html());
		
//		List<Url> urls = urlDbUtils.getAllUrls();
//		int size = urls.size();
//		for (int i = 0; i <size; i++) {
//			Url url = urls.get(i);
//			LOGGER.debug(url.getRemark()+","+url.getHref());
//		}
		
		
//		List<AdministraRegion> ars = administraRegionDbUtils.getByGrade(1);
//		int size = ars.size();
//		for (int i = 0; i < size ; i++) {
//			AdministraRegion ar = ars.get(i);
//			List<AdministraRegion> subordinates = administraRegionDbUtils.getAllSubordinateById(ar.getId());
//			int ssize = subordinates.size();
//			for (int j = 0; j < ssize; j++) {
//				AdministraRegion sar = subordinates.get(j);
//				System.out.println(sar.getId()+","+sar.getName());
//			}
//			Integer urlId = ar.getUrlId();
//			LOGGER.debug("url id:"+urlId);
//			if(urlId!=null) {
//				Url url = urlDbUtils.getById(urlId);
//				Document doc = Downloader.downloadByGet(url.getHref(), GBK);
//				Elements es = doc.select("div[class=namecity] a");
//				if (!es.isEmpty()) {
//					int essize = es.size();
//					for (int j = 0; j < essize; j++) {
//						Element e = es.get(j);
//						String href = URL_PREFIX+e.attr("href");
//						String tn = e.html();
//						Integer arId = getSubordinateId(subordinates,tn);
//						if (arId!=-1) {
//							Document fdoc2 = Downloader.downloadByGet(href, GBK);
//							Elements ies = fdoc2.select("iframe[class=irw]");
//							if (!ies.isEmpty()) {
//								href = URL_PREFIX+ies.first().attr("src");
//								Url url2 = new Url();
//								url2.setHref(href);
//								url2.setRemark(tn);
//								url2.setStatus(1);
//								url2.setUrlTypeId(1);
//								Integer uId = urlDbUtils.add(url2);
//								administraRegionDbUtils.updateUrlId(arId, uId);
//							}
//						}
//					}
//				}
//				
//				System.out.println(doc.html());
//				
//			}
//			break;
//		}
		
//		String url6 = "http://www.nmc.gov.cn/publish/forecast/ABJ/shunyi_iframe.html";
//		Document doc6 = Downloader.downloadByGet(url6, GBK);
//		Elements es6 = doc6.select("div[class=city_wind]");
//		if (!es6.isEmpty()) {
//			String info = es6.first().html();
//			System.out.println(info);
//			info = info.replaceAll("<br\\s*/>","|").replaceAll("<.*?>", "").replaceAll("[\\s\n]+","");
//			System.out.println(info);
//			String[] infos = info.split("\\|");
//			int length = infos.length;
//			for (int i = 0; i < length; i++) {
//				System.out.println(infos[i]);
//			}
//		}
		
//		String url7 = "http://www.nmc.gov.cn/publish/weather/provice/ABJ_24.html";
//		Document doc7 = Downloader.downloadByGet(url7, GBK);
//		System.out.println(doc7.html());
	}

}
